from NewsMining.db.dbConnect import getDBConnection
from mynewsweb.model import *
from math import sqrt
from NewsMining.crawler.preprocessText import *
from NewsMining.db import dbConnect
from cache import cached_kernel
import orange
import orngSVM
import orngTest
import orngStat
from orngSVM import KernelWrapper
import numpy as np
from sqlalchemy import and_
import sys
import time
import testData

from svm import *

print "Start:", time.asctime()
out = open("results/Ngram4.txt", "a")
#==== ==== ==== ====
# Settings
#==== ==== ==== ====
try:
    numTests = int(sys.argv[1])
    k = int(sys.argv[2])
    sample_length = int(sys.argv[3])
    runOrange, runDB = 0, 0
except:
    numTests = 1
    k = 10
    sample_length = 200
    runOrange, runDB = 0, 0

#==== ==== ==== ====
# CLEAR RESULTS
#==== ==== ==== ====
db = getDBConnection()

#==== ==== ==== ====
# NGRAM KERNEL
#==== ==== ==== ====
def ngram_kernel(s1, s2, data, k=0, decay=0):
    '''simple n-gram string kernel'''
    s1d = data[s1]
    s2d = data[s2]
    inter = set(s1d.keys()).intersection(s2d.keys())

    return sum(s1d[i] * s2d[i] for i in inter)

def ngram_dot_kernel(n1, n2, data, k=0, decay=0):
    return np.dot(data[n1], data[n2])

#==== ==== ==== ====
# ORANGE SVM
#==== ==== ==== ====
print "Create Orange examples"

# Data in orange format
domainDef = [orange.EnumVariable(name="class_id", values=["1", "-1"])]
domain = orange.Domain(domainDef)

news_id_var = orange.StringVariable("news_id")
rank_var = orange.FloatVariable("rank")

id = orange.newmetaid()
rank_id = orange.newmetaid()

domain.addmeta(id, news_id_var)
domain.addmeta(rank_id, rank_var)

#==== ==== ==== ====
# DATA
#==== ==== ==== ====
print "Prepare data!"

news = testData.getAllData(sample_length, random=False)
#news = testData.getData(sample_length)

print "Positive:", sum([1 for i in news if i.class_id == 1])
print "Negative:", sum([1 for i in news if i.class_id == -1])

news_data = {}
examples = []
svm_data = []

# ==== PREPARE ====
# GENERATE Ngrams FOR News + ALL GRAMS = S'
all_ngrams = {}

for n in news:
    text = n.content_normalized
    if len(text) > 300:
        kmers = [text[i:i+k] for i in range(len(text)-k+1)]
        d = dict([(i, 0) for i in set(kmers)])
        #svm_d = dict([(toInt(i), 0) for i in set(kmers)])

        for i in kmers:
            d[i] += 1
            #svm_d[toInt(i)] += 1
            # Top 1000
            if all_ngrams.has_key(i):
                all_ngrams[i] += 1
            else:
                all_ngrams[i] = 1

        news_data[n.news_id] = d


        # Orange
        if runOrange:
            o = [str(n.class_id)]
            ex = orange.Example(domain, o)
            ex['news_id'] = str(n.news_id)
            ex['rank'] = str(n.rank)
            #print o
            examples.append(ex)

# TOP 1000 freqvent ngrams
print "TOP 1000 freqvent ngrams"
print "all_ngrams length = %s" % len(all_ngrams)
sgrams = all_ngrams.items()
sgrams.sort(lambda x,y: -cmp(x[1], y[1]))
sgrams = sgrams[:6000]
d = dict([(i[0], 0) for i in sgrams])

# Calculate News with Top 1000 ngrams
print "Calculate News with Top 1000 ngrams"
news_produkt = {}

for n in news:
    if n.news_id  in news_data:
        ngrams = news_data[n.news_id]
        #news_sgrams = d.copy()
        news_produkt[n.news_id] = [ngrams.get(i, 0) for i in d]
        svm_data.append((n.class_id, news_produkt[n.news_id]))

#==== ==== ==== ====
# REAL SVM
#==== ==== ==== ====
#==== F-MEASURE ====
def fMeasure(precision, recall):
    try:
        return 2 * ((precision * recall) / (precision + recall))
    except:
        print "error at fmeasure", precision, recall
        return 0.5

# ==== AUC ====
def auc(result):
    result = sorted(result, lambda x, y: -cmp(x[1][1][1], y[1][1][1]))
    for r in result:
        print r[0], "\t", r[1][0], "\t", round(r[1][1][1], 2)
    ploscina = x = y = 0

    for i in result:
        if i[0] == 1:
            x += 1
        else:
            ploscina += x
            y += 1
    try:
        return ploscina / float(x * y)
    except:
        print "error at auc", ploscina, x, y
        return 0.5


folds = 4
prop = [0.2, 0.4, 0.8, 1.0]
C = 10
weight = [1, 4] # 1, 4 teh best
#prediction_results = {}
#orange_results = {}
##
####for C in [1, 3, 4, 5, 8, 10, 20]:
####    for posWeight in range(1, 10, 2):
##for p in prop:
##    #weight = [1, posWeight]
##    print "\n" * 3
##    print p
##    print "#" * 20
##    print "C: %s, %s" % (C, weight)
##    print "#" * 20
##
##    ca_array = np.array([])
##    auc_array = np.array([])
##    f1_array = np.array([])
##    # ==== PREPARE DATA ====
##    prop_data = svm_data[:int(len(svm_data) * p)]
##
##    slices = [prop_data[i::k] for i in xrange(folds)]
##
##    # ==== FOLDS ====
##    for fold in range(folds):
##        print fold, "=" * 10
##        # ==== TRAIN and TEST DATA ====
##        test_data = slices[fold]
##        train_data = [i for slice in slices if slice is not test_data for i in slice]
##
##        train_labels = [i[0] for i in train_data]
##        train_samples = [i[1] for i in train_data]
##
##        prediction_results = []
##        orange_results = []
##
##        # ==== MODEL ====
##        problem = svm_problem(train_labels, train_samples)
##        param = svm_parameter(kernel_type= LINEAR, C = C, nr_weight = 2,
##                            weight_label = [-1, 1], weight = weight, probability = 1)
##        model = svm_model(problem,param)
##
##        # ==== MEASURES ====
##        correct = truePositive = myPositive = allPositive = 0
##        ca = precision = recall = 0
##        ex = orngTest.ExperimentResults(2, ['LIN'], [1, -1], -1)
##
##        # ==== PREDICTION ====
##        for i in test_data:
##            probability = model.predict_probability(i[1])
##            prediction = probability[0]
##            # Save prediction
##            prediction_results.append((i[0], probability))
##            t = orngTest.TestedExample(0, i[0])
##            t.probabilities = [[probability[1][-1], probability[1][1]]]
##            ex.results.append(t)
##
##            # ca
##            if i[0] == prediction:
##                correct += 1
##            # precision, Kolikokrat sem rekel, da ima nekdo bolezen
##            if prediction == 1:
##                myPositive += 1
##                # Kolikokrat sem pravilno rekel, da ima nekdo bolezen
##                if i[0] == prediction == 1:
##                    truePositive += 1
##            # recall, Kolikokrat bi moral, reci, da ima nekdo bolezen
##            if i[0] == 1:
##                allPositive += 1
##
##        # ==== CALCULATE RESULTS ====
##        try:
##            ca = correct / float(len(test_data))
##            precision = truePositive / float(myPositive)
##            recall = truePositive / float(allPositive)
##            f1 = fMeasure(precision, recall)
##            auc_ = auc(prediction_results)
##            auc_orange = orngStat.AUC(ex)[0]
##        except:
##            print "error" * 10
##            ca = precision = recall = f1 = auc_ = auc_orange = 0.5
##
##        ca_array = np.append(ca_array, ca)
##        auc_array = np.append(auc_array, auc_)
##        f1_array = np.append(f1_array, f1)
##
##        # ==== PRINT RESULTS ====
##        print "*" * 10
##        print p, fold
##        print "CA:", ca
##        print "Precision:", precision
##        print "recall:", recall
##        print "F1:", f1
##        print "AUC:", auc_
##        print "AUC_orange:", auc_orange
##
##    print ca_array
##    print auc_array
##    print f1_array
##
##    results = [round(i, 3) for i in ca_array.mean(), auc_array.mean(), auc_orange, f1_array.mean()]
##    toPrint = map(str, [k, p, len(svm_data) * p, C, weight] + results)
##
##    out.write("\t".join(toPrint) + "\n")
##    out.flush()
##
##out.write("\n")

# ==========================================
# ==========================================
# ==========================================
print "#" * 20
print "C: %s, %s" % (C, weight)
print "#" * 20

# ===== Predict for one weak ====
print "*" * 20
test_size = 100
train_size = 200

for index in range(0, 3000, test_size):
    if index + train_size + test_size < len(svm_data):

        ca_array = np.array([])
        auc_array = np.array([])
        f1_array = np.array([])
        # ==== TRAIN and TEST DATA ====
        train_data = svm_data[index: index + train_size]
        test_data = svm_data[index + train_size: index + (train_size + test_size)]

        print " ==== Dolzine ===="
        print len(train_data)
        print len(test_data)

        train_labels = [i[0] for i in train_data]
        train_samples = [i[1] for i in train_data]

        prediction_results = []
        orange_results = []

        # ==== MODEL ====
        problem = svm_problem(train_labels, train_samples)
        param = svm_parameter(kernel_type= LINEAR, C = C, nr_weight = 2,
                            weight_label = [-1, 1], weight = weight, probability = 1)
        model = svm_model(problem,param)

        # ==== MEASURES ====
        correct = truePositive = myPositive = allPositive = 0
        ca = precision = recall = 0
        ex = orngTest.ExperimentResults(2, ['LIN'], [1, -1], -1)

        # ==== PREDICTION ====
        for i in test_data:
            probability = model.predict_probability(i[1])
            prediction = probability[0]
            # Save prediction
            prediction_results.append((i[0], probability))
            t = orngTest.TestedExample(0, i[0])
            t.probabilities = [[probability[1][-1], probability[1][1]]]
            ex.results.append(t)

            # ca
            if i[0] == prediction:
                correct += 1
            # precision, Kolikokrat sem rekel, da ima nekdo bolezen
            if prediction == 1:
                myPositive += 1
                # Kolikokrat sem pravilno rekel, da ima nekdo bolezen
                if i[0] == prediction == 1:
                    truePositive += 1
            # recall, Kolikokrat bi moral, reci, da ima nekdo bolezen
            if i[0] == 1:
                allPositive += 1

        # ==== CALCULATE RESULTS ====
        try:
            precision = truePositive / float(myPositive)
            recall = truePositive / float(allPositive)
        except:
            print "error" * 10
            precision = recall = 0.5

        ca = correct / float(len(test_data))
        f1 = fMeasure(precision, recall)
        auc_ = auc(prediction_results)
        auc_orange = orngStat.AUC(ex)[0]

        ca_array = np.append(ca_array, ca)
        auc_array = np.append(auc_array, auc_)
        f1_array = np.append(f1_array, f1)

        test_poz = [i for i in test_data if i[0] == 1]
        test_neg = [i for i in test_data if i[0] == -1]

        # ==== PRINT RESULTS ====
        print "*" * 10
        print "Index:", index
        print "CA:", ca
        print "Precision:", precision
        print "recall:", recall
        print "F1:", f1
        print "AUC:", auc_
        print "AUC_orange:", auc_orange

        print ca_array
        print auc_array
        print f1_array

        results = [round(i, 3) for i in ca_array.mean(), auc_array.mean(), auc_orange, f1_array.mean()]
        toPrint = map(str, [k, index, index + train_size, len(test_poz), len(test_neg), C, weight] + results)

        out.write("\t".join(toPrint) + "\n")
        out.flush()

out.write("\n")

#==== ==== ==== ====
# CALL
#==== ==== ==== ====
kernel = cached_kernel(ngram_dot_kernel) #ngram_kernel

#ALL vs ALL
if runDB:
    print "Clear previus results"
    db.execute("delete from   kernel_results where  kernel_id = 1")
    db.commit()

    print "Run db"
    id_row = db.execute("SELECT max(id_row) + 1 FROM kernel_results").fetchone()[0]
    id_row = [0, id_row][id_row != None]

    i = 0
    for n1 in news_data.keys()[:]:
        print n1
        for n2 in news_data:
            if n1 != n2:
                d = kernel(n1, n2, news_data) / sqrt(kernel(n1, n1, news_data) * kernel(n2, n2, news_data))
                if d > 0:
                    #r = KernelResults(n1, n2, d, 1, k, 0)
                    #db.add(r)
                    db.execute("INSERT INTO kernel_results VALUES (%s, %s, %s, %s, %s, %s, %s)" % (id_row, n1, n2, d, 1, k, 0))
                    id_row += 1

        i += 1
        if i % 300 == 0:
            print "Commit"
            db.commit()

    db.commit()


#==== ==== ==== ====
# SVM - kernel
#==== ==== ==== ====
if runOrange:
    print "Run SVM!!!"
    data = orange.ExampleTable(examples)

    def intermediate_kernel(n1, n2):
        return kernel(n1, n2, news_data) / sqrt(kernel(n1, n1, news_data) * kernel(n2, n2, news_data))

    def intermediate_dot_kernel(n1, n2):
        return kernel(n1, n2, news_produkt) / sqrt(kernel(n1, n1, news_produkt) * kernel(n2, n2, news_produkt))

    def example_kernel(example1, example2):
        return intermediate_dot_kernel(int(example1['news_id'].value), int(example2['news_id'].value))

    # ==== Build SVM ====
    l1=orngSVM.SVMLearner() #weight=[("-1", 1), ("1", 1)]
    l1.kernelFunc=KernelWrapper(example_kernel) #orngSVM. #Linear
    l1.kernel_type=orange.SVMLearner.Custom
    l1.probability=True
    l1.name="LIN"

    l2=orngSVM.SVMLearner()
    l2.kernelFunc=orngSVM.RBFKernelWrapper(example_kernel, gamma=0.5)
    l2.kernel_type=orange.SVMLearner.Custom
    l2.probability=True
    l2.name="RBF"

    l3=orngSVM.SVMLearner()
    l3.kernelFunc=orngSVM.PolyKernelWrapper(example_kernel, degree=3.0)
    l3.kernel_type=orange.SVMLearner.Custom
    l3.probability=True
    l3.name="POLY"

    print "*" * 10
    print "SVM: N-grams"
    print " "
    #print "\t".join(["k", "prop", "ex"] + [l.name + "\tauc\tf1\tf0.5" for l in [l1, l2, l3][:numTests]])

    # ==== LEARNING CURVE =====
    prop = [0.04, 0.1, 0.2, 0.4, 0.5, 0.6, 0.8, 1.0]
    res = orngTest.learningCurveN([l1, l2, l3][:numTests], data, folds = 2, proportions = prop)
    for i, p in enumerate(prop):

        ca, auc = orngStat.CA(res[i]), orngStat.AUC(res[i])
        confusionMatrica = orngStat.confusionMatrices(res[i])
        fMeasure, fAlpha = orngStat.F1(confusionMatrica), orngStat.Falpha(confusionMatrica, 0.5)

        results = [round(i, 3) for j in zip(ca, auc, fMeasure, fAlpha) for i in j]
        toPrint = map(str, [k, p, int(len(news) * p)] + results)

        print "\t".join(toPrint)
        out.write("\t".join(toPrint) + "\n")
        out.flush()


##    # ===== Predict for one weak ====
##    print "*" * 20
##    data.sort('rank')
##    test_size = 50
##    train_size = 250
##
##    for index in range(0, 1000, test_size):
##        if index + train_size + test_size < len(data):
##            train = data.getitems(range(0, index + train_size))
##            test = data.getitems(range(index + train_size, index + (train_size + test_size)))
##            res = orngTest.learnAndTestOnTestData([l1, l2, l3][:numTests], train, test)
##
##            print "\n\n"
##            print "#" * 10
##            for i, j in zip(res.results, test):
##                pass
##                #print i.actualClass, i.probabilities[0], j
##            print "#" * 10
##
##            print "==== TRAIN ===="
##            print "Pozitive:", sum([1 for i in train if i['class_id'].value == "1"])
##            print "Negative:", sum([1 for i in train if i['class_id'].value == "-1"])
##
##            print "==== TEST ===="
##            print "Pozitive:", sum([1 for i in test if i['class_id'].value == "1"])
##            print "Negative:", sum([1 for i in test if i['class_id'].value == "-1"])
##
##            ca, auc = orngStat.CA(res), orngStat.AUC(res)
##            confusionMatrica = orngStat.confusionMatrices(res)
##            fMeasure, fAlpha = orngStat.F1(confusionMatrica), orngStat.Falpha(confusionMatrica, 0.5)
##
##            results = [round(i, 3) for j in zip(ca, auc, fMeasure, fAlpha) for i in j]
##            toPrint = map(str, [k, train[0]['rank'], train[-1]['rank'], test[0]['rank'], test[-1]['rank']] + results)
##            #print "Train: ", train[0]['rank'], train[-1]['rank'], ", Test:", test[0]['rank'], test[-1]['rank']
##            print "\t".join(toPrint)
##            out.write("\t".join(toPrint) + "\n")

out.close()
print "*" * 20
print "End:", time.asctime()

